﻿using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Diagnostics.Eventing.Reader;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Dapper;
using DotnetSpider.DataFlow;
using DotnetSpider.DataFlow.Parser;
using DotnetSpider.DataFlow.Storage;
using DotnetSpider.Http;
using DotnetSpider.RequestSupplier;
using DotnetSpider.Scheduler.Component;
using DotnetSpider.Selector;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;

using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RabbitMQ.Client;
using Serilog;
using StringContent = DotnetSpider.Http.StringContent;

namespace DotnetSpider.Sample.samples
{
	public static class CommonHelper
	{

		public static string NoHtml(string Htmlstring)
		{
			if (Htmlstring == null)
			{
				return "";
			}
			else
			{
				//删除脚本
				Htmlstring = Regex.Replace(Htmlstring, @"<script[^>]*?>.*?</script>", "", RegexOptions.IgnoreCase);
				//删除HTML
				Htmlstring = Regex.Replace(Htmlstring, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"([\r\n])[\s]+", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"-->", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"<!--.*", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(quot|#34);", "\"", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(amp|#38);", "&", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(lt|#60);", "<", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(gt|#62);", ">", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(nbsp|#160);", " ", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(iexcl|#161);", "\xa1", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(cent|#162);", "\xa2", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(pound|#163);", "\xa3", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&(copy|#169);", "\xa9", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, @"&#(\d+);", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "xp_cmdshell", "", RegexOptions.IgnoreCase);

				//删除与数据库相关的词
				Htmlstring = Regex.Replace(Htmlstring, "select", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "insert", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "delete from", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "count''", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "drop table", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "truncate", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "asc", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "mid", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "char", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "xp_cmdshell", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "exec master", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "net localgroup administrators", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "and", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "net user", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "or", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "net", "", RegexOptions.IgnoreCase);
				//Htmlstring = Regex.Replace(Htmlstring, "*", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "-", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "delete", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "drop", "", RegexOptions.IgnoreCase);
				Htmlstring = Regex.Replace(Htmlstring, "script", "", RegexOptions.IgnoreCase);

				//特殊的字符
				Htmlstring = Htmlstring.Replace("<", "");
				Htmlstring = Htmlstring.Replace(">", "");
				Htmlstring = Htmlstring.Replace("*", "");
				Htmlstring = Htmlstring.Replace("-", "");
				Htmlstring = Htmlstring.Replace("?", "");
				Htmlstring = Htmlstring.Replace("'", "''");
				Htmlstring = Htmlstring.Replace(",", "");
				Htmlstring = Htmlstring.Replace("/", "");
				Htmlstring = Htmlstring.Replace(";", "");
				Htmlstring = Htmlstring.Replace("*/", "");
				Htmlstring = Htmlstring.Replace("\r\n", "");
				Htmlstring = WebUtility.HtmlEncode(Htmlstring);

				return Htmlstring;
			}
		}




	}
}
